{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from neural_networks import NeuralNetwork\n",
    "from layers import RNN,Activation\n",
    "from losses import CrossEntropy\n",
    "from optimizers import StochasticGradientDescent\n",
    "from progress import to_categorical,train_test_split\n",
    "from metric import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "optimizer = StochasticGradientDescent()\n",
    "\n",
    "def gen_mult_ser(nums):\n",
    "    \"\"\" Method which generates multiplication series \"\"\"\n",
    "    X = np.zeros([nums, 10, 61], dtype=float)\n",
    "    y = np.zeros([nums, 10, 61], dtype=float)\n",
    "    for i in range(nums):\n",
    "        start = np.random.randint(2, 7)\n",
    "        mult_ser = np.linspace(start, start*10, num=10, dtype=int)\n",
    "        X[i] = to_categorical(mult_ser, n_col=61)\n",
    "        y[i] = np.roll(X[i], -1, axis=0)\n",
    "    y[:, -1, 1] = 1 # Mark endpoint as 1\n",
    "    return X, y\n",
    "def gen_num_seq(nums):\n",
    "    \"\"\" Method which generates sequence of numbers \"\"\"\n",
    "    X = np.zeros([nums, 10, 20], dtype=float)\n",
    "    y = np.zeros([nums, 10, 20], dtype=float)\n",
    "    for i in range(nums):\n",
    "        start = np.random.randint(0, 10)\n",
    "        num_seq = np.arange(start, start+10)\n",
    "        X[i] = to_categorical(num_seq, n_col=20)\n",
    "        y[i] = np.roll(X[i], -1, axis=0)\n",
    "    y[:, -1, 1] = 1 # Mark endpoint as 1\n",
    "    return X, y\n",
    "X, y = gen_mult_ser(3000)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = NeuralNetwork(optimizer=optimizer,\n",
    "                    loss=CrossEntropy)\n",
    "clf.add(RNN(10, activation=\"tanh\", bptt_trunc=5, input_shape=(10, 61)))\n",
    "clf.add(Activation('softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number Series Problem:\n",
      "X = [3 6 9 12 15 18 21 24 27 30]\n",
      "y = [6 9 12 15 18 21 24 27 30 1]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tmp_X = np.argmax(X_train[0], axis=1)\n",
    "tmp_y = np.argmax(y_train[0], axis=1)\n",
    "print (\"Number Series Problem:\")\n",
    "print (\"X = [\" + \" \".join(tmp_X.astype(\"str\")) + \"]\")\n",
    "print (\"y = [\" + \" \".join(tmp_y.astype(\"str\")) + \"]\")\n",
    "print ()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training: 100% [------------------------------------------------] Time: 0:01:21\n"
     ]
    }
   ],
   "source": [
    "train_err, _ = clf.fit(X_train, y_train, n_epochs=500, batch_size=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Results:\n",
      "X      = [5 10 15 20 25 30 35 40 45 50]\n",
      "y_true = [10 15 20 25 30 35 40 45 50 1]\n",
      "y_pred = [10 15 27 25 30 35 40 45 50 1]\n",
      "\n",
      "X      = [3 6 9 12 15 18 21 24 27 30]\n",
      "y_true = [6 9 12 15 18 21 24 27 30 1]\n",
      "y_pred = [6 9 12 15 18 21 24 27 32 3]\n",
      "\n",
      "X      = [5 10 15 20 25 30 35 40 45 50]\n",
      "y_true = [10 15 20 25 30 35 40 45 50 1]\n",
      "y_pred = [10 15 27 25 30 35 40 45 50 1]\n",
      "\n",
      "X      = [3 6 9 12 15 18 21 24 27 30]\n",
      "y_true = [6 9 12 15 18 21 24 27 30 1]\n",
      "y_pred = [6 9 12 15 18 21 24 27 32 3]\n",
      "\n",
      "X      = [3 6 9 12 15 18 21 24 27 30]\n",
      "y_true = [6 9 12 15 18 21 24 27 30 1]\n",
      "y_pred = [6 9 12 15 18 21 24 27 32 3]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "y_pred = np.argmax(clf.predict(X_test), axis=2)\n",
    "y_test = np.argmax(y_test, axis=2)\n",
    "\n",
    "print ()\n",
    "print (\"Results:\")\n",
    "for i in range(5):\n",
    "    # Print a problem instance and the correct solution\n",
    "    tmp_X = np.argmax(X_test[i], axis=1)\n",
    "    tmp_y1 = y_test[i]\n",
    "    tmp_y2 = y_pred[i]\n",
    "    print (\"X      = [\" + \" \".join(tmp_X.astype(\"str\")) + \"]\")\n",
    "    print (\"y_true = [\" + \" \".join(tmp_y1.astype(\"str\")) + \"]\")\n",
    "    print (\"y_pred = [\" + \" \".join(tmp_y2.astype(\"str\")) + \"]\")\n",
    "    print ()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy = np.mean(accuracy_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.65825"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
